from time import sleep
import datetime
import re

import requests
from lxml import etree
from pymongo import MongoClient
from selenium.webdriver import Firefox

from freeProxyWebs.proxyweb import ProxyWebs

class KuaiProxy(ProxyWebs):
    ##获取快代理的代理ip    https://www.kuaidaili.com/free/

    def _get_web(self, url):
        return super()._get_web(url)

    def _parser(self,html):
        ##解析网页，得到代理列表，如果代理不是当天，返回False
        Today = True
        tree = etree.HTML(html)
        treeRowList = tree.xpath('//table/tbody/tr')
        for treeRow in treeRowList:
            infos = treeRow.xpath('td/text()')
            ip = infos[0]
            port = infos[1]
            protocol = infos[3]
            proxytime = re.findall('[0-9]{4}-[01][0-9]-[0123][0-9]',infos[-1])[0]
            proxy = {'ip':ip,'port':port,'protocol':protocol}
            self._proxyList.append(proxy)
            proxyTime = datetime.datetime.strptime(
                proxytime,'%Y-%m-%d'
            )   #代理更新时间
            yestardayTime = datetime.datetime.now() \
                + datetime.timedelta(days= -1)  #昨日时间
            if proxyTime < yestardayTime:
                ##如果代理更新时间早于昨天，Today设为假
                Today = False
        return Today

    def refresh_daily_proxy(self):
        ##更新每日代理
        maxPage = 10    #最多爬的页数
        i = 1
        utypes = ['inha/']
        for utype in utypes:
            urlr = self._url + utype
            for i in range(1,maxPage+1):
                url = urlr + f'{i}/'
                Today = self._parser(self._get_web(url))
                if not Today:
                    break
                sleep(1)
    
    def init_proxy_pool(self):
        ##初始化代理池，爬取前20页的代理
        url = self._url
        utypes = ['inha/']
        for utype in utypes:
            urlr = self._url + utype
            for i in range(1,21):
                url = urlr + f'{i}/'
                self._parser(self._get_web(url))
                sleep(1)
        #将保存的集合初始化
        client = MongoClient('mongodb://localhost:27017/')
        with client:
            db = client.ipPool
            try:
                db.kuai.drop()
            except:
                print('没有集合，直接创建')
            finally:
                db.kuai.insert_many(self._proxyList)
                print("快代理集合初始化完成")

    def save_proxies(self):
        ##将代理列表保存入数据库
        client = MongoClient('mongodb://localhost:27017/')
        with client:
            db = client.ipPool
            db.kuai.insert_many(self._proxyList)
            print("快代理保存完成")

